import pandas as pd
import py_stringsimjoin as ssj
import py_stringmatching as sm
import sys
import time
import py_entitymatching as em
from py_entitymatching.catalog import catalog_manager as cm

def preprocess_title(title):
    title = title.lower()#小写
    title = title.replace(',', ' ')
    title = title.replace("'", '')
    title = title.replace("\"",'')#增加 移除双引号
    title = title.replace('(','')
    title = title.replace(')','')
    title = title.replace('（','')
    title = title.replace('）','')
    title = title.replace('&', 'and')
    title = title.replace(";", '')
    title = title.replace(' ','')
    #title = title.encode('utf-8', 'ignore')
    return title.strip()

# baseinfo reginfo_extract3(modify) 匹配的两个表名
# 0.8 筛选候选集的阈值
# 2 匹配的两对属性
# name authorNationality
# company_id company_id
# 1 需要规范化的字符串属性对数
# name authorNationality
# 2 mixture
# nor_name nor_authorNationality
# company_id company_id
#候选集中左表和右表分别要保留的项
#3
#company_id nor_name credit_code
#4
#company_id nor_authorNationality fullname regnum


def main(argv):
    len=argv.__len__()
    # for j in range(len):
        # print(sys.argv[j])
    start = time.clock()  # 时间计算开始
    # 读取数据表
    if(argv[1].find('swj')>=0):
        argv[1]+='_normal'
        # print("add normal",argv[1])
    if(argv[2].find('swj')>=0):
        argv[2]+='_normal'
        # print("add normal", argv[2])
    csvpath1 = "D:/eclipse-workspace/DataIntergration/python/schema-matching-master/demo/dy/"+argv[1]+".csv"
    csvpath2 = "D:/eclipse-workspace/DataIntergration/python/schema-matching-master/demo/dy/"+argv[2]+".csv"
    data1 = pd.read_csv(csvpath1, encoding='utf-8')
    data2 = pd.read_csv(csvpath2, encoding='utf-8')

    thresholdinput = float(argv[3])
    matchpairnum = int(argv[4])

    matchpair = [[]]*matchpairnum
    for i in range(matchpairnum):
        matchpair[i]=[argv[5+2*i],argv[6+2*i]]

    temp=7+2*i
    norm_num=int(argv[temp])
    normschema=[[]]*norm_num
    normname=[[]]*norm_num
    for i in range(norm_num):
        normschema[i] = [argv[temp+2*i+1],argv[temp+2*i+2]]
        # print(normschema[i])
        normname[i] = ['nor_'+argv[temp+2*i+1],'nor_'+argv[temp+2*i+2]]
        data1[normname[i][0]] = data1[normschema[i][0]].map(preprocess_title)#表1规范化
        data2[normname[i][1]] = data2[normschema[i][1]].map(preprocess_title)#表2规范化

    temp2=temp+2*i+3
    mixture_num=int(argv[temp2])
    mixtureschema=[[]]*mixture_num
    for i in range(mixture_num):
        mixtureschema[i]=[argv[temp2+1+2*i],argv[temp2+2+2*i]]
    # print(mixtureschema)
    temp3=temp2+3+2*i



    data1['id'] = range(data1.shape[0])#增加了一列用于id标识
    data2['id'] = range(data2.shape[0])

    #Magellan
    #Substep A: Finding a candidate set (Blocking)

    # transforming the "company_id" column into string
    for i in range(matchpairnum):#不管怎样 将所有的转化为str
        ssj.utils.converter.dataframe_column_to_str(data1, matchpair[i][0], inplace=True)#将数字的转换成str
        ssj.utils.converter.dataframe_column_to_str(data2, matchpair[i][1], inplace=True)

    # creating a new **mixture** column
    data1['mixture']=data1[mixtureschema[0][0]]
    data2['mixture']=data2[mixtureschema[0][1]]
    for i in range(1,mixture_num):
        data1['mixture']+=''+data1[mixtureschema[i][0]]#字符串的累加
        data2['mixture']+=''+data2[mixtureschema[i][1]]

    l_out_num = int(argv[temp3])
    l_out_atrrss = [argv[temp3 + 1]]
    for i in range(1, l_out_num):
        l_out_atrrss.append(argv[temp3 + 1 + i])
    # print("l_out_attrs", l_out_atrrss)
    temp4 = temp3 + 2 + i
    r_out_num = int(argv[temp4])
    r_out_attrss = [argv[temp4 + 1]]
    for i in range(1, r_out_num):
        r_out_attrss.append(argv[temp4 + 1 + i])
    # print("r_out_attrs", r_out_attrss)


    # print(data2['mixture'])
    #Qgram分词设定一定的阈值，提取候选集
    C1 = ssj.overlap_coefficient_join(data1, data2, 'id', 'id', 'mixture', 'mixture',sm.QgramTokenizer(),
                                     l_out_attrs=l_out_atrrss, r_out_attrs=r_out_attrss,
                                     threshold=thresholdinput)
    C_name="C_"+argv[1]+"_"+argv[2]
    C1.to_csv("D:/eclipse-workspace/DataIntergration/WebContent/output/"+C_name+".csv",sep=',',encoding="utf-8",index = False)
    print("C1.shape：",C1.shape)
    print("C_name:",C_name)


    samplenum=C1.shape[0]//10
    if(argv[1].find('swj')>=0&argv[2].find('swj')>=0):
        samplenum=150
    print("label:",samplenum)
    # Sampling 500 pairs and writing this sample into a .csv file 提取样本集，以待标注
    sampled = C1.sample(samplenum, random_state=0)
    sampled['label']=1
    label_name="labeled_"+argv[1]+"_"+argv[2]
    print("label_name:", label_name)
    sampled.to_csv('D:/eclipse-workspace/DataIntergration/WebContent/output/'+label_name+'.csv', encoding='utf-8')


if __name__ == '__main__':
  sys.exit(main(sys.argv))








